{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:32.409004Z",
     "iopub.status.busy": "2024-10-08T02:38:32.408247Z",
     "iopub.status.idle": "2024-10-08T02:38:32.436481Z",
     "shell.execute_reply": "2024-10-08T02:38:32.434183Z",
     "shell.execute_reply.started": "2024-10-08T02:38:32.408934Z"
    },
    "ExecuteTime": {
     "end_time": "2024-10-08T03:18:36.859170Z",
     "start_time": "2024-10-08T03:18:36.857222Z"
    }
   },
   "source": [
    "%env LLM_API_KEY=替换为自己的key\n",
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LLM_API_KEY=替换为自己的key\n",
      "env: LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:33.213465Z",
     "iopub.status.busy": "2024-10-08T02:38:33.212722Z",
     "iopub.status.idle": "2024-10-08T02:38:36.182918Z",
     "shell.execute_reply": "2024-10-08T02:38:36.182509Z",
     "shell.execute_reply.started": "2024-10-08T02:38:33.213397Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "langchain                     0.2.10\n",
      "langchain_core                0.2.28\n",
      "langchain_community           0.2.9\n",
      "pypdf                         4.3.1\n",
      "sentence_transformers         3.0.1\n",
      "chromadb                      0.5.4\n"
     ]
    }
   ],
   "source": [
    "import chromadb\n",
    "import langchain\n",
    "import langchain_community\n",
    "import langchain_core\n",
    "import pypdf\n",
    "import sentence_transformers\n",
    "\n",
    "for module in (langchain, langchain_core, langchain_community, pypdf, sentence_transformers, chromadb):\n",
    "    print(f\"{module.__name__:<30}{module.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:36.949472Z",
     "iopub.status.busy": "2024-10-08T02:38:36.947166Z",
     "iopub.status.idle": "2024-10-08T02:38:36.957504Z",
     "shell.execute_reply": "2024-10-08T02:38:36.955288Z",
     "shell.execute_reply.started": "2024-10-08T02:38:36.949391Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:37.278369Z",
     "iopub.status.busy": "2024-10-08T02:38:37.277579Z",
     "iopub.status.idle": "2024-10-08T02:38:37.283089Z",
     "shell.execute_reply": "2024-10-08T02:38:37.282673Z",
     "shell.execute_reply.started": "2024-10-08T02:38:37.278299Z"
    }
   },
   "outputs": [],
   "source": [
    "expr_version = 'retrieval_v8_step_back_prompting'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:39.213880Z",
     "iopub.status.busy": "2024-10-08T02:38:39.213073Z",
     "iopub.status.idle": "2024-10-08T02:38:40.783178Z",
     "shell.execute_reply": "2024-10-08T02:38:40.782694Z",
     "shell.execute_reply.started": "2024-10-08T02:38:39.213808Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()\n",
    "\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:41.268384Z",
     "iopub.status.busy": "2024-10-08T02:38:41.267247Z",
     "iopub.status.idle": "2024-10-08T02:38:41.276924Z",
     "shell.execute_reply": "2024-10-08T02:38:41.276429Z",
     "shell.execute_reply.started": "2024-10-08T02:38:41.268307Z"
    }
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:38:41.954323Z",
     "iopub.status.busy": "2024-10-08T02:38:41.952797Z",
     "iopub.status.idle": "2024-10-08T02:38:41.966522Z",
     "shell.execute_reply": "2024-10-08T02:38:41.964175Z",
     "shell.execute_reply.started": "2024-10-08T02:38:41.954243Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found cache, restoring...\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(preprocess_output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:04.053697Z",
     "iopub.status.busy": "2024-10-08T02:39:04.052913Z",
     "iopub.status.idle": "2024-10-08T02:39:04.346182Z",
     "shell.execute_reply": "2024-10-08T02:39:04.345726Z",
     "shell.execute_reply.started": "2024-10-08T02:39:04.053626Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "# llm = ChatOllama(base_url='http://localhost:11434', model='qwen2:7b-instruct')\n",
    "# 使用Ollama，注意，不要使用上面这行，直接使用ChatOllama类后续流程会报错\n",
    "llm = ChatOpenAI(\n",
    "    model='qwen2:7b-instruct', \n",
    "    base_url='http://localhost:11434/v1', \n",
    "    api_key='ollama'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:05.838265Z",
     "iopub.status.busy": "2024-10-08T02:39:05.837481Z",
     "iopub.status.idle": "2024-10-08T02:39:08.495879Z",
     "shell.execute_reply": "2024-10-08T02:39:08.495420Z",
     "shell.execute_reply.started": "2024-10-08T02:39:05.838195Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseChatModel.__call__` was deprecated in langchain-core 0.1.7 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "AIMessage(content='我是通义千问，是由阿里云开发的一个AI助手。我被设计用来回答各种问题、提供信息和与用户进行对话。有什么特定的问题或主题你想要了解更多吗？我会尽力帮助你。', response_metadata={'token_usage': {'completion_tokens': 46, 'prompt_tokens': 10, 'total_tokens': 56}, 'model_name': 'qwen2:7b-instruct', 'system_fingerprint': 'fp_ollama', 'finish_reason': 'stop', 'logprobs': None}, id='run-e6e537a1-151d-41c7-984b-2cc4f07ed1cf-0', usage_metadata={'input_tokens': 10, 'output_tokens': 46, 'total_tokens': 56})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm('你是谁')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-09T16:39:38.174798Z",
     "iopub.status.busy": "2024-08-09T16:39:38.174362Z",
     "iopub.status.idle": "2024-08-09T16:39:38.179679Z",
     "shell.execute_reply": "2024-08-09T16:39:38.178235Z",
     "shell.execute_reply.started": "2024-08-09T16:39:38.174731Z"
    }
   },
   "source": [
    "下面这段是官方示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:10.655400Z",
     "iopub.status.busy": "2024-10-08T02:39:10.654587Z",
     "iopub.status.idle": "2024-10-08T02:39:10.670319Z",
     "shell.execute_reply": "2024-10-08T02:39:10.668522Z",
     "shell.execute_reply.started": "2024-10-08T02:39:10.655329Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "\n",
    "system = \"\"\"You are an expert at taking a specific question and extracting a more generic question that gets at \\\n",
    "the underlying principles needed to answer the specific question.\n",
    "\n",
    "You will be asked about a set of software for building LLM-powered applications called LangChain, LangGraph, LangServe, and LangSmith.\n",
    "\n",
    "LangChain is a Python framework that provides a large set of integrations that can easily be composed to build LLM applications.\n",
    "LangGraph is a Python package built on top of LangChain that makes it easy to build stateful, multi-actor LLM applications.\n",
    "LangServe is a Python package built on top of LangChain that makes it easy to deploy a LangChain application as a REST API.\n",
    "LangSmith is a platform that makes it easy to trace and test LLM applications.\n",
    "\n",
    "Given a specific user question about one or more of these products, write a more generic question that needs to be answered in order to answer the specific question. \\\n",
    "\n",
    "If you don't recognize a word or acronym to not try to rewrite it.\n",
    "\n",
    "Write concise questions.\"\"\"\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\"system\", system),\n",
    "        (\"human\", \"{question}\"),\n",
    "    ]\n",
    ")\n",
    "# llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
    "step_back = prompt | llm | StrOutputParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:10.928175Z",
     "iopub.status.busy": "2024-10-08T02:39:10.927403Z",
     "iopub.status.idle": "2024-10-08T02:39:10.957307Z",
     "shell.execute_reply": "2024-10-08T02:39:10.954970Z",
     "shell.execute_reply.started": "2024-10-08T02:39:10.928106Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ChatPromptValue(messages=[SystemMessage(content=\"You are an expert at taking a specific question and extracting a more generic question that gets at the underlying principles needed to answer the specific question.\\n\\nYou will be asked about a set of software for building LLM-powered applications called LangChain, LangGraph, LangServe, and LangSmith.\\n\\nLangChain is a Python framework that provides a large set of integrations that can easily be composed to build LLM applications.\\nLangGraph is a Python package built on top of LangChain that makes it easy to build stateful, multi-actor LLM applications.\\nLangServe is a Python package built on top of LangChain that makes it easy to deploy a LangChain application as a REST API.\\nLangSmith is a platform that makes it easy to trace and test LLM applications.\\n\\nGiven a specific user question about one or more of these products, write a more generic question that needs to be answered in order to answer the specific question. \\nIf you don't recognize a word or acronym to not try to rewrite it.\\n\\nWrite concise questions.\"), HumanMessage(content='美国单一家庭房贷整体拖欠率在2023年二季度达到多少？')])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt.invoke({'question': '美国单一家庭房贷整体拖欠率在2023年二季度达到多少？'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:11.906872Z",
     "iopub.status.busy": "2024-10-08T02:39:11.906310Z",
     "iopub.status.idle": "2024-10-08T02:39:12.622904Z",
     "shell.execute_reply": "2024-10-08T02:39:12.622536Z",
     "shell.execute_reply.started": "2024-10-08T02:39:11.906844Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2023年第二季度全美单户家庭的抵押贷款拖欠率是多少?'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "step_back.invoke({'question': '美国单一家庭房贷整体拖欠率在2023年二季度达到多少？'})"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备Step back prompt"
   ]
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "可以使用官方的chain，也可以自己定义prompt，此处为了讲清楚原理，使用自定义的chain"
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:42.016403Z",
     "iopub.status.busy": "2024-10-08T02:39:42.016207Z",
     "iopub.status.idle": "2024-10-08T02:39:42.021048Z",
     "shell.execute_reply": "2024-10-08T02:39:42.020536Z",
     "shell.execute_reply.started": "2024-10-08T02:39:42.016388Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import FewShotChatMessagePromptTemplate, ChatPromptTemplate\n",
    "\n",
    "examples = [\n",
    "    {\n",
    "        \"input\": \"美联储预计下半年利率会下调多少？\",\n",
    "        \"output\": \"美联储下半年的利率政策会如何制定\",\n",
    "    },\n",
    "    {\n",
    "        \"input\": \"如何评估一家公司股票的内在价值？\",\n",
    "        \"output\": \"什么方法可以用来评估资产的真实价值？\",\n",
    "    }\n",
    "]\n",
    "# 转换为消息\n",
    "example_prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        ('human', '{input}'),\n",
    "        ('ai', '{output}')\n",
    "    ]\n",
    ")\n",
    "few_shot_prompt = FewShotChatMessagePromptTemplate(\n",
    "    example_prompt=example_prompt,\n",
    "    examples=examples\n",
    ")\n",
    "prompt = ChatPromptTemplate.from_messages([\n",
    "    (\n",
    "        'system',\n",
    "        \"\"\"你是金融领域的专家。你的任务是把一个问题改写成一个更一般或更抽象的问题，但注意仅问题改写得更一般即可，如果问题有主体，问题的主体要保持不变。如果你不确定如何改写，请保持原问题不变。直接输出改写后的问题即可，不需要包含“改写之后的问题”之类的描述性内容。这里有几个例子:\"\"\"\n",
    "    ),\n",
    "    few_shot_prompt,\n",
    "    ('user', '{question}')\n",
    "])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备用于生成更抽象问题的Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:42.709235Z",
     "iopub.status.busy": "2024-10-08T02:39:42.708420Z",
     "iopub.status.idle": "2024-10-08T02:39:42.721890Z",
     "shell.execute_reply": "2024-10-08T02:39:42.719655Z",
     "shell.execute_reply.started": "2024-10-08T02:39:42.709165Z"
    }
   },
   "outputs": [],
   "source": [
    "step_back_query_gen = (\n",
    "    prompt\n",
    "    | llm\n",
    "    | (lambda ai_msg: ai_msg.content)\n",
    "    | (lambda x: x.replace('AI:', '').replace('ai:', '').strip() if x.lower().startswith('ai:') else x)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:43.481117Z",
     "iopub.status.busy": "2024-10-08T02:39:43.480948Z",
     "iopub.status.idle": "2024-10-08T02:39:44.321796Z",
     "shell.execute_reply": "2024-10-08T02:39:44.321330Z",
     "shell.execute_reply.started": "2024-10-08T02:39:43.481104Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'什么是影响个人贷款违约率的主要因素？'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "step_back_query_gen.invoke({'question': '美国单一家庭房贷整体拖欠率在2023年二季度达到多少？'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备检索器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:50.470856Z",
     "iopub.status.busy": "2024-10-08T02:39:50.470681Z",
     "iopub.status.idle": "2024-10-08T02:39:50.480928Z",
     "shell.execute_reply": "2024-10-08T02:39:50.480487Z",
     "shell.execute_reply.started": "2024-10-08T02:39:50.470843Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "import shutil\n",
    "import torch\n",
    "\n",
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "# 可以替换为本地模型地址\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings\n",
    "\n",
    "\n",
    "def get_vector_db(embeddings, docs, db_name):\n",
    "    persist_directory = os.path.join(expr_dir, 'chroma', db_name)\n",
    "    shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "\n",
    "    vector_db = Chroma.from_documents(\n",
    "        splitted_docs,\n",
    "        embedding=embeddings,\n",
    "        persist_directory=persist_directory\n",
    "    )\n",
    "    return vector_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:39:51.653799Z",
     "iopub.status.busy": "2024-10-08T02:39:51.653010Z",
     "iopub.status.idle": "2024-10-08T02:40:06.633799Z",
     "shell.execute_reply": "2024-10-08T02:40:06.632832Z",
     "shell.execute_reply.started": "2024-10-08T02:39:51.653728Z"
    }
   },
   "outputs": [],
   "source": [
    "vector_db = get_vector_db(get_embeddings(model_path), splitted_docs, 'step_back')\n",
    "\n",
    "def get_retrieve_chain(top_k):\n",
    "    chain = (\n",
    "        step_back_query_gen\n",
    "        | (lambda x: vector_db.similarity_search(x, k=top_k))\n",
    "    )\n",
    "    return chain"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:06.635475Z",
     "iopub.status.busy": "2024-10-08T02:40:06.635193Z",
     "iopub.status.idle": "2024-10-08T02:40:06.639631Z",
     "shell.execute_reply": "2024-10-08T02:40:06.638793Z",
     "shell.execute_reply.started": "2024-10-08T02:40:06.635448Z"
    }
   },
   "outputs": [],
   "source": [
    "retrieve_chain = get_retrieve_chain(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:06.640964Z",
     "iopub.status.busy": "2024-10-08T02:40:06.640656Z",
     "iopub.status.idle": "2024-10-08T02:40:07.395735Z",
     "shell.execute_reply": "2024-10-08T02:40:07.395339Z",
     "shell.execute_reply.started": "2024-10-08T02:40:06.640935Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'page': 33, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '3e312dfa-dd43-4ab9-961f-a2e442c89cdd'}, page_content='全球经济金融展望报告\\n中国银行研究院 32 2024年\\n图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3：全球主要股指概览\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
       " Document(metadata={'page': 51, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'ebf0d999-59f6-4fd3-941e-05a7a60c255a'}, page_content='免责声明\\n本研究报告由中国银行研究院撰写，研究报告中所引用信息均来自公开资料。\\n本研究报告中包含的观点或估计仅代表作者迄今为止的判断，它们不一定反映中国银行的观点。中国\\n银行研究院可以不经通知加以改变，且没有对此报告更新、修正或修改的责任。\\n本研究报告内容及观点仅供参考，不构成任何投资建议。对于本报告所提供信息所导致的任何直接的\\n或者间接的投资盈亏后果不承担任何责任。\\n本研究报告版权仅为中国银行研究院所有，未经书面许可，任何机构和个人不得以任何形式翻版、复\\n制和发布。如引用发布，需注明出处为中国银行研究院，且不得对本报告进行有悖原意的引用、删节和修\\n改。中国银行研究院保留对任何侵权行为和有悖报告原意的引用行为进行追究的权利。'),\n",
       " Document(metadata={'page': 0, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'e73a0c9d-d42b-4350-a4c3-b38bf67c68a5'}, page_content='研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12月12日\\n●2023年全球经济增长动力持续回落，各国复苏分化，\\n发达经济体增速明显放缓，新兴经济体整体表现稳定。\\n全球贸易增长乏力，各国生产景气度逐渐回落，内需\\n对经济的拉动作用减弱。欧美央行货币政策紧缩态势\\n放缓，美元指数高位震荡后走弱，全球股市表现总体\\n好于预期，但区域分化明显。高利率环境抑制债券融\\n资需求，债券违约风险持续上升。\\n●展望2024年，预计全球经济复苏将依旧疲软，主要\\n经济体增长态势和货币政策走势将进一步分化。欧美\\n央行大概率结束本轮紧缩货币周期，美元指数将逐步\\n走弱，流向新兴经济体的跨境资本将增加。国际原油\\n市场短缺格局或延续，新能源发展成为重点。\\n●海湾六国经济发展与投资前景、高利率和高债务对\\n美国房地产市场脆弱性的影响等热点问题值得关注。中国银行研究院\\n全球经济金融研究课题组\\n组长：陈卫东\\n副组长：钟红\\n廖淑萍\\n成员：边卫红\\n熊启跃\\n王有鑫\\n曹鸿宇\\n李颖婷\\n王宁远\\n初晓\\n章凯莉\\n黄小军（纽约）\\n陆晓明（纽约）\\n黄承煜（纽约）\\n宋达志（伦敦）\\n李振龙（伦敦）\\n张传捷（伦敦）\\n刘冰彦（法兰克福）\\n温颍坤（法兰克福）\\n张明捷（法兰克福）\\n王哲（东京）\\n李彧（香港）\\n黎永康（香港）\\n联系人：王有鑫\\n电话：010-66594127\\n邮件：wangyouxin_hq@bank-of-china.com主要经济体GDP增速变化趋势（%）\\n资料来源：IMF，中国银行研究院')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieve_chain.invoke('报告的发布机构是什么？')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 检索性能对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:10.639120Z",
     "iopub.status.busy": "2024-10-08T02:40:10.638302Z",
     "iopub.status.idle": "2024-10-08T02:40:10.652106Z",
     "shell.execute_reply": "2024-10-08T02:40:10.649892Z",
     "shell.execute_reply.started": "2024-10-08T02:40:10.639049Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = qa_df[(qa_df['dataset'] == 'test') & (qa_df['qa_type'] == 'detailed')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 不使用Step Back Prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:12.507059Z",
     "iopub.status.busy": "2024-10-08T02:40:12.506715Z",
     "iopub.status.idle": "2024-10-08T02:40:12.514030Z",
     "shell.execute_reply": "2024-10-08T02:40:12.513212Z",
     "shell.execute_reply.started": "2024-10-08T02:40:12.507032Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "def get_hit_stat_df(top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "    pbar = tqdm(total=len(top_k_arr) * len(test_df))\n",
    "    for k in top_k_arr:\n",
    "        pbar.set_description(f'k={k}')\n",
    "\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "\n",
    "            chunks = vector_db.similarity_search(question, k=k)\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks]\n",
    "            assert len(chunks) <= k\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "            pbar.update(1)\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:13.798717Z",
     "iopub.status.busy": "2024-10-08T02:40:13.797935Z",
     "iopub.status.idle": "2024-10-08T02:40:31.074151Z",
     "shell.execute_reply": "2024-10-08T02:40:31.073702Z",
     "shell.execute_reply.started": "2024-10-08T02:40:13.798647Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b61e8f4164430da7e3a5b6e9af388a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/744 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "orig_query_hit_stat_df = get_hit_stat_df()\n",
    "orig_query_hit_stat_df['step_back_prompting'] = 'w/o'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用Step Back Prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:51.166857Z",
     "iopub.status.busy": "2024-10-08T02:40:51.166655Z",
     "iopub.status.idle": "2024-10-08T02:40:51.170801Z",
     "shell.execute_reply": "2024-10-08T02:40:51.170359Z",
     "shell.execute_reply.started": "2024-10-08T02:40:51.166843Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_step_back_hit_stat_df(top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "    pbar = tqdm(total=len(top_k_arr) * len(test_df))\n",
    "    for k in top_k_arr:\n",
    "        pbar.set_description(f'k={k}')\n",
    "\n",
    "        step_back_retrieve_chain = get_retrieve_chain(k)\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            \n",
    "            chunks = step_back_retrieve_chain.invoke(question)\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks]\n",
    "            \n",
    "            assert len(chunks) <= k\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "            pbar.update(1)\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:40:51.570059Z",
     "iopub.status.busy": "2024-10-08T02:40:51.569265Z",
     "iopub.status.idle": "2024-10-08T02:51:54.975852Z",
     "shell.execute_reply": "2024-10-08T02:51:54.975384Z",
     "shell.execute_reply.started": "2024-10-08T02:40:51.569988Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14f9777762264c4197091c9190c092e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/744 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "step_back_hit_stat_df = get_step_back_hit_stat_df()\n",
    "step_back_hit_stat_df['step_back_prompting'] = 'w/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:58:02.135670Z",
     "iopub.status.busy": "2024-10-08T02:58:02.134870Z",
     "iopub.status.idle": "2024-10-08T02:58:02.145806Z",
     "shell.execute_reply": "2024-10-08T02:58:02.145029Z",
     "shell.execute_reply.started": "2024-10-08T02:58:02.135599Z"
    }
   },
   "outputs": [],
   "source": [
    "hit_stat_df = pd.concat([orig_query_hit_stat_df, step_back_hit_stat_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:58:02.482694Z",
     "iopub.status.busy": "2024-10-08T02:58:02.482296Z",
     "iopub.status.idle": "2024-10-08T02:58:02.490456Z",
     "shell.execute_reply": "2024-10-08T02:58:02.490015Z",
     "shell.execute_reply.started": "2024-10-08T02:58:02.482677Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step_back_prompting</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>w/</td>\n",
       "      <td>1</td>\n",
       "      <td>0.301075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>w/</td>\n",
       "      <td>2</td>\n",
       "      <td>0.505376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w/</td>\n",
       "      <td>3</td>\n",
       "      <td>0.526882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>w/</td>\n",
       "      <td>4</td>\n",
       "      <td>0.645161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>w/</td>\n",
       "      <td>5</td>\n",
       "      <td>0.655914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>w/</td>\n",
       "      <td>6</td>\n",
       "      <td>0.741935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>w/</td>\n",
       "      <td>7</td>\n",
       "      <td>0.720430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>w/</td>\n",
       "      <td>8</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>w/o</td>\n",
       "      <td>1</td>\n",
       "      <td>0.462366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>w/o</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>w/o</td>\n",
       "      <td>3</td>\n",
       "      <td>0.688172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>w/o</td>\n",
       "      <td>4</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>w/o</td>\n",
       "      <td>5</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>w/o</td>\n",
       "      <td>6</td>\n",
       "      <td>0.817204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>w/o</td>\n",
       "      <td>7</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>w/o</td>\n",
       "      <td>8</td>\n",
       "      <td>0.849462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   step_back_prompting  top_k  hit_rate\n",
       "0                   w/      1  0.301075\n",
       "1                   w/      2  0.505376\n",
       "2                   w/      3  0.526882\n",
       "3                   w/      4  0.645161\n",
       "4                   w/      5  0.655914\n",
       "5                   w/      6  0.741935\n",
       "6                   w/      7  0.720430\n",
       "7                   w/      8  0.784946\n",
       "8                  w/o      1  0.462366\n",
       "9                  w/o      2  0.591398\n",
       "10                 w/o      3  0.688172\n",
       "11                 w/o      4  0.774194\n",
       "12                 w/o      5  0.806452\n",
       "13                 w/o      6  0.817204\n",
       "14                 w/o      7  0.838710\n",
       "15                 w/o      8  0.849462"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hit_stat_df.groupby(['step_back_prompting', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:58:03.086094Z",
     "iopub.status.busy": "2024-10-08T02:58:03.085312Z",
     "iopub.status.idle": "2024-10-08T02:58:03.522192Z",
     "shell.execute_reply": "2024-10-08T02:58:03.521710Z",
     "shell.execute_reply.started": "2024-10-08T02:58:03.086025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='top_k', ylabel='hit'>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4c0lEQVR4nO3dfVRVVf7H8c8FBUR5EBFQRMmHUEyhgXDQMalIRhvLcUapLAyNZkrK4jemlEFqivaAmDmSD6RTOVozZlamFQZlqTiIjpZiNimkAjqVKCYo8PvD5Z1INFS453J8v9Y6a3H33fue7waKj/vsc6+ltra2VgAAACbhYHQBAAAAjYlwAwAATIVwAwAATIVwAwAATIVwAwAATIVwAwAATIVwAwAATIVwAwAATKWF0QXYWk1NjQ4dOiQ3NzdZLBajywEAAA1QW1ur48ePq2PHjnJwuPjazFUXbg4dOqSAgACjywAAAJehuLhYnTp1umifqy7cuLm5STr7zXF3dze4GgAA0BDl5eUKCAiw/h2/mKsu3Jy7FOXu7k64AQCgmWnIlhI2FAMAAFMh3AAAAFMh3AAAAFO56vbcNFR1dbVOnz5tdBm4irRs2VKOjo5GlwEAzR7h5mdqa2tVUlKiH374wehScBXy9PSUn58f78EEAFeAcPMz54KNj4+PXF1d+SMDm6itrdXJkydVVlYmSerQoYPBFQFA80W4+Ynq6mprsGnXrp3R5eAq06pVK0lSWVmZfHx8uEQFAJeJDcU/cW6Pjaurq8GV4Gp17neP/V4AcPkIN/XgUhSMwu8eAFw5wg0AADAVwk0zFhUVpUcffbRJz3Hfffdp+PDhTXqO5sAW32sAQONgQ3EjuO+++/TDDz9o9erVRpeCK5STk6ObbrpJ33//vTw9Pa3tq1atUsuWLY0rDADQYIQb2JWqqio5OTkZXcZ5vLy8jC4BANBAXJa6BP/4xz/Up08ftWrVSu3atVN0dLQmTpyoZcuW6e2335bFYpHFYlFOTo4kqbi4WKNGjZKnp6e8vLx0xx13aP/+/dbXO3fJZ+rUqWrfvr3c3d315z//WVVVVQ2u6cyZM0pMTJSHh4e8vb311FNPqba21vr8q6++qvDwcLm5ucnPz09333239b1Uzvniiy/0u9/9Tu7u7nJzc9PAgQP19ddf13u+rVu3qn379po9e/Yv1vb0008rNDRUL7/8sgICAuTq6qpRo0bp2LFj530PZsyYoY4dOyooKEiStHPnTt18883W7/UDDzygEydOnDdu5syZ8vX1laenp6ZNm6YzZ85o4sSJ8vLyUqdOnfTKK69Yx+zfv18Wi0UrVqxQ//795eLiouuuu065ubnW52+66SZJUtu2bWWxWHTfffdJOv+yVGBgoGbOnKmxY8fKzc1NnTt31sKFC+vM//PPP1doaKhcXFwUHh6u1atXy2KxaPv27b/4vQMAXD7CTQMdPnxYd911l8aOHavdu3crJydHI0aMUGpqqkaNGqXf/va3Onz4sA4fPqz+/fvr9OnTiomJkZubmz799FN99tlnatOmjX7729/WCS/Z2dnW1/v73/+uVatWaerUqQ2ua9myZWrRooXy8vI0d+5cpaena/HixdbnT58+renTp2vHjh1avXq19u/fb/2DLUkHDx7UjTfeKGdnZ23YsEH5+fkaO3aszpw5c965NmzYoFtvvVUzZszQpEmTGlTfvn379MYbb+idd97RunXrVFBQoIceeqhOn+zsbBUWFurDDz/Uu+++q4qKCsXExKht27baunWr3nzzTX300UdKTEw8r55Dhw7pk08+UXp6ulJTU/W73/1Obdu21ZYtW/TnP/9Zf/rTn/Ttt9/WGTdx4kT93//9nwoKChQZGalhw4bpv//9rwICAvTPf/5TklRYWKjDhw9r7ty5F5zbCy+8oPDwcOucHnzwQRUWFkqSysvLNWzYMPXp00fbtm3T9OnTG/w9AwBcGS5LNdDhw4d15swZjRgxQl26dJEk9enTR9LZN1+rrKyUn5+ftf9rr72mmpoaLV682Hp77yuvvCJPT0/l5ORo8ODBkiQnJydlZWXJ1dVVvXv31rRp0zRx4kRNnz5dDg6/nD0DAgI0Z84cWSwWBQUFaefOnZozZ44SEhIkSWPHjrX27dq1q1588UXdcMMNOnHihNq0aaP58+fLw8NDK1assO4pufbaa887z1tvvaW4uDgtXrxYsbGxDf6+nTp1Sn/729/k7+8vSZo3b55uu+02vfDCC9bvV+vWrbV48WLr5ahFixZZx7Vu3VqS9NJLL2nYsGGaPXu2fH19JZ29VPTiiy/KwcFBQUFBevbZZ3Xy5Ek98cQTkqTk5GTNmjVLGzdu1J133mmtKTExUX/4wx8kSQsWLNC6deu0ZMkSPf7449bLTz4+PnX23NRn6NCh1qA2adIkzZkzRx9//LGCgoK0fPlyWSwWLVq0SC4uLgoODtbBgwetPxcAaAphE//W5OfIfy6uyc9xpVi5aaCQkBDdcsst6tOnj0aOHKlFixbp+++/v2D/HTt2aN++fXJzc1ObNm3Upk0beXl56dSpU3Uu+YSEhNR508DIyEidOHFCxcXFDarr17/+dZ33RomMjNRXX32l6upqSVJ+fr6GDRumzp07y83NTYMGDZIkFRUVSZK2b9+ugQMHXnSz7JYtWzRy5Ei9+uqrlxRsJKlz587WYHOuvpqaGusKh3Q2JP50n83u3bsVEhJiDTaSNGDAgPPG9e7du04A9PX1tQZOSXJ0dFS7du3OuwwXGRlp/bpFixYKDw/X7t27L2lektS3b1/r1xaLRX5+ftZzFRYWqm/fvnJxcbH2iYiIuORzAAAuHeGmgRwdHfXhhx/q/fffV3BwsObNm6egoCB988039fY/ceKEwsLCtH379jrH3r17dffdd9uk5nOXd9zd3fX6669r69ateuuttyTJemns3Fv+X0y3bt3Us2dPZWVlNck75/40xFyKnwcyi8VSb1tNTc1l13ap52+qcwEAGo5wcwksFosGDBigqVOnqqCgQE5OTnrrrbfk5ORkXSk551e/+pW++uor+fj4qHv37nUODw8Pa78dO3boxx9/tD7evHmz2rRpo4CAgAbVtGXLljqPN2/erB49esjR0VF79uzRf//7X82aNUsDBw5Uz549z1vF6Nu3rz799NOLhhZvb29t2LBB+/bt06hRoy4p4BQVFenQoUN16jt3GelCevXqpR07dqiiosLa9tlnn/3iuIbavHmz9eszZ84oPz9fvXr1kiTrCtLPf56X6twlwsrKSmvb1q1br+g1AQANw56bBtqyZYuys7M1ePBg+fj4aMuWLTpy5Ih69eqlU6dOaf369SosLFS7du3k4eGh0aNH67nnntMdd9yhadOmqVOnTjpw4IBWrVqlxx9/XJ06dZJ0dgVl3LhxmjJlivbv36/U1FQlJiY2aL+NdDY8JCUl6U9/+pO2bdumefPm6YUXXpB09pKQk5OT5s2bpz//+c/atWuXpk+fXmd8YmKi5s2bpzvvvFPJycny8PDQ5s2bFRERUSdI+Pj4aMOGDbrpppt01113acWKFWrR4pd/fVxcXDRmzBg9//zzKi8v1yOPPKJRo0bV2Z/0c6NHj1ZqaqrGjBmjp59+WkeOHNHDDz+se++917rf5krMnz9fPXr0UK9evTRnzhx9//331r1JXbp0kcVi0bvvvquhQ4eqVatWatOmzSWf4+6779aTTz6pBx54QJMnT1ZRUZGef/55SXzEAmAL7D25urFy00Du7u765JNPNHToUF177bWaMmWKXnjhBQ0ZMkQJCQkKCgpSeHi42rdvr88++0yurq765JNP1LlzZ40YMUK9evXSuHHjdOrUKbm7u1tf95ZbblGPHj104403KjY2VrfffruefvrpBtcVFxenH3/8URERERo/frwmTJigBx54QJLUvn17LV26VG+++aaCg4M1a9Ys6x/Yc9q1a6cNGzboxIkTGjRokMLCwrRo0aJ69+D4+flpw4YN2rlzp0aPHt2g1Y3u3btrxIgRGjp0qAYPHqy+ffvqr3/960XHuLq6av369fruu+90ww036I9//KNuueUWvfTSSw3+vlzMrFmzNGvWLIWEhGjjxo1as2aNvL29JUn+/v6aOnWqJk+eLF9f3/Pu0Good3d3vfPOO9q+fbtCQ0P15JNPKiUlRZLq7MMBADQ+S+1P3xTlKlBeXi4PDw8dO3asTsiQzt7Z88033+iaa66xyR8gs7+z8dNPP63Vq1fbzfu67N+/X9dcc40KCgoUGhpq8/O//vrrio+P17Fjxy6418nWv4OAWV2tKzdmnvfF/n7/HJelgCbyt7/9TV27dpW/v7927NihSZMmadSoUQ3axA0AuHyEGztVVFSk4ODgCz7/5ZdfqnPnzjas6Hy9e/fWgQMH6n3u5ZdftnE19qekpEQpKSkqKSlRhw4dNHLkSM2YMcPosgDA9Ag3Blq6dOkFn+vYseNFL+d07Nix8Qu6RGvXrr3gnVO+vr5yc3O7pP1DTS0wMFC2vAr7+OOP6/HHH7fZ+QAAZxFu7FSLFi3UvXt3o8u4qHPv1AwAgD3hbikAAGAqhBsAAGAqXJYCABMz863BwIWwcgMAAEyFcAMAAEzF8HAzf/58BQYGysXFRf369VNeXt5F+2dkZCgoKEitWrVSQECAHnvsMZ06dcpG1QIAAHtn6J6blStXKikpSZmZmerXr58yMjIUExOjwsJC+fj4nNd/+fLlmjx5srKystS/f3/t3btX9913nywWi9LT0w2YgX2yxTX2n2rq6+25ubm65557VFxc3KTngbmx9wS4ehi6cpOenq6EhATFx8crODhYmZmZcnV1VVZWVr39P//8cw0YMEB33323AgMDNXjwYN11112/uNqD5u3tt9/WsGHDjC4DANBMGBZuqqqqlJ+fr+jo6P8V4+Cg6Ohobdq0qd4x/fv3V35+vjXM/Oc//9HatWs1dOjQC56nsrJS5eXldQ4Y591335Wnp6f1E8W3b98ui8WiyZMnW/vcf//9uueee6yP16xZo9tvv13S2Z/nI488Ih8fH7m4uOg3v/mNtm7dattJAADsmmHh5ujRo6qurpavr2+ddl9fX5WUlNQ75u6779a0adP0m9/8Ri1btlS3bt0UFRWlJ5544oLnSUtLk4eHh/UICAho1Hng0gwcOFDHjx9XQUGBpLOXnLy9vZWTk2Ptk5ubq6ioKEnSF198obKyMt18882Szn6kwT//+U8tW7ZM27ZtU/fu3RUTE6PvvvvO1lMBANgpwzcUX4qcnBzNnDlTf/3rX7Vt2zatWrVK7733nqZPn37BMcnJyTp27Jj1YN+GsTw8PBQaGmoNMzk5OXrsscdUUFCgEydO6ODBg9q3b58GDRok6ewlqZiYGDk5OamiokILFizQc889pyFDhig4OFiLFi1Sq1attGTJEgNnBQCwJ4aFG29vbzk6Oqq0tLROe2lpqfz8/Ood89RTT+nee+/V/fffrz59+uj3v/+9Zs6cqbS0NNXU1NQ7xtnZWe7u7nUOGGvQoEHKyclRbW2tPv30U40YMUK9evXSxo0blZubq44dO6pHjx6Szoabc5ekvv76a50+fVoDBgywvlbLli0VERGh3bt3GzIXAID9MSzcODk5KSwsTNnZ2da2mpoaZWdnKzIyst4xJ0+elIND3ZIdHR0lyaaf9owrExUVpY0bN2rHjh1q2bKlevbsqaioKOXk5Cg3N9e6anP48GEVFBTotttuM7hiAEBzYuhlqaSkJC1atEjLli3T7t279eCDD6qiokLx8fGSpLi4OCUnJ1v7Dxs2TAsWLNCKFSv0zTff6MMPP9RTTz2lYcOGWUMO7N+5fTdz5syxBplz4SYnJ8e63+add95R//795eXlJUnq1q2bnJyc9Nlnn1lf6/Tp09q6dauCg4NtPg8AgH0y9H1uYmNjdeTIEaWkpKikpEShoaFat26ddZNxUVFRnZWaKVOmyGKxaMqUKTp48KDat2+vYcOGacaMGUZNAZehbdu26tu3r15//XW99NJLkqQbb7xRo0aN0unTp62B56d3SUlS69at9eCDD2rixIny8vJS586d9eyzz+rkyZMaN26cIXMBANgfwz84MzExUYmJifU+99M7aCSpRYsWSk1NVWpqqg0qQ1MaNGiQtm/fbl2l8fLyUnBwsEpLSxUUFKSKigplZ2crIyOjzrhZs2appqZG9957r44fP67w8HCtX79ebdu2tf0kAOAqVDStT5Ofo3PKzisab3i4QeNrDu+SmpGRcV5w2b59u/Xr9evX65prrlH37t3r9HFxcdGLL76oF1980QZVAgCao2Z1KziuHm3atNHs2bONLgMA0AyxcgO7NHjwYKNLAAA0U6zcAAAAU2HlBrjK8OnYAMyOlRsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAq3ApuQrb43I+futLPAPklubm5uueee1RcXNyk5wEAmAMrN7B7b7/9toYNG2Z0GQCAZoJwA5t699135enpqerqaklnPyzTYrFo8uTJ1j7333+/7rnnHuvjNWvW6Pbbb7d5rQCA5olwA5saOHCgjh8/roKCAklnLzl5e3srJyfH2ic3N1dRUVGSpC+++EJlZWW6+eabDagWANAcEW5gUx4eHgoNDbWGmZycHD322GMqKCjQiRMndPDgQe3bt0+DBg2SdPaSVExMjJycnAysGgDQnBBuYHODBg1STk6Oamtr9emnn2rEiBHq1auXNm7cqNzcXHXs2FE9evSQdDbccEkKAHApuFsKNhcVFaWsrCzt2LFDLVu2VM+ePRUVFaWcnBx9//331lWbw4cPq6CgQLfddpvBFQO4GFvcodnUd2XCXFi5gc2d23czZ84ca5A5F25ycnKs+23eeecd9e/fX15eXgZWCwBobgg3sLm2bduqb9++ev31161B5sYbb9S2bdu0d+9ea+DhLikAwOUg3MAQgwYNUnV1tTXceHl5KTg4WH5+fgoKClJFRYWys7MJNwCAS8aeGxNqDtemMzIylJGRUadt+/bt1q/Xr1+va665Rt27d7dtYQCAZo+VG9ilNm3aaPbs2UaXAQBohli5gV0aPHiw0SUAAJopVm4AAICpEG4AAICpcFmqHrW1tUaXgKsUv3tA88GbF9ovVm5+omXLlpKkkydPGlwJrlbnfvfO/S4CAC4dKzc/4ejoKE9PT5WVlUmSXF1dZbFYDK4KV4Pa2lqdPHlSZWVl8vT0lKOjo9ElAUCzRbj5GT8/P0myBhzAljw9Pa2/gwCAy2MX4Wb+/Pl67rnnVFJSopCQEM2bN08RERH19o2KilJubu557UOHDtV77713xbVYLBZ16NBBPj4+On369BW/HtBQLVu2ZMUGABqB4eFm5cqVSkpKUmZmpvr166eMjAzFxMSosLBQPj4+5/VftWqVqqqqrI//+9//KiQkRCNHjmzUuhwdHflDAwBAM2T4huL09HQlJCQoPj5ewcHByszMlKurq7Kysurt7+XlJT8/P+vx4YcfytXVtdHDDQAAaJ4MDTdVVVXKz89XdHS0tc3BwUHR0dHatGlTg15jyZIluvPOO9W6deumKhMAADQjhl6WOnr0qKqrq+Xr61un3dfXV3v27PnF8Xl5edq1a5eWLFlywT6VlZWqrKy0Pi4vL7/8ggEAgN0z/LLUlViyZIn69Olzwc3HkpSWliYPDw/rERAQYMMKAQCArRkabry9veXo6KjS0tI67aWlpb94O2xFRYVWrFihcePGXbRfcnKyjh07Zj2Ki4uvuG4AAGC/DA03Tk5OCgsLU3Z2trWtpqZG2dnZioyMvOjYN998U5WVlbrnnnsu2s/Z2Vnu7u51DgAAYF6G3wqelJSkMWPGKDw8XBEREcrIyFBFRYXi4+MlSXFxcfL391daWlqdcUuWLNHw4cPVrl07I8oGAAB2yvBwExsbqyNHjiglJUUlJSUKDQ3VunXrrJuMi4qK5OBQd4GpsLBQGzdu1AcffGBEyTCJsIl/a/Jz5D8X1+TnAADUZXi4kaTExEQlJibW+1xOTs55bUFBQXx6MgAAqFezvlsKAADg5wg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVAg3AADAVOzigzMBwAyKpvVp8nN0TtnZ5OcAmjtWbgAAgKkQbgAAgKkQbgAAgKmw5wZAo2PvCQAjsXIDAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMxfBwM3/+fAUGBsrFxUX9+vVTXl7eRfv/8MMPGj9+vDp06CBnZ2dde+21Wrt2rY2qBQAA9q6FkSdfuXKlkpKSlJmZqX79+ikjI0MxMTEqLCyUj4/Pef2rqqp06623ysfHR//4xz/k7++vAwcOyNPT0/bFAwAAu2RouElPT1dCQoLi4+MlSZmZmXrvvfeUlZWlyZMnn9c/KytL3333nT7//HO1bNlSkhQYGGjLkgEAgJ0z7LJUVVWV8vPzFR0d/b9iHBwUHR2tTZs21TtmzZo1ioyM1Pjx4+Xr66vrrrtOM2fOVHV1ta3KBgAAds6wlZujR4+qurpavr6+ddp9fX21Z8+eesf85z//0YYNGzR69GitXbtW+/bt00MPPaTTp08rNTW13jGVlZWqrKy0Pi4vL2+8SQAAALtj+IbiS1FTUyMfHx8tXLhQYWFhio2N1ZNPPqnMzMwLjklLS5OHh4f1CAgIsGHFAADA1gwLN97e3nJ0dFRpaWmd9tLSUvn5+dU7pkOHDrr22mvl6OhobevVq5dKSkpUVVVV75jk5GQdO3bMehQXFzfeJAAAgN0xLNw4OTkpLCxM2dnZ1raamhplZ2crMjKy3jEDBgzQvn37VFNTY23bu3evOnToICcnp3rHODs7y93dvc4BAADMy9DLUklJSVq0aJGWLVum3bt368EHH1RFRYX17qm4uDglJydb+z/44IP67rvvNGHCBO3du1fvvfeeZs6cqfHjxxs1BQAAYGcMvRU8NjZWR44cUUpKikpKShQaGqp169ZZNxkXFRXJweF/+SsgIEDr16/XY489pr59+8rf318TJkzQpEmTjJqCKYRN/FuTnyP/ubgmPwcAAJLB4UaSEhMTlZiYWO9zOTk557VFRkZq8+bNTVwVAABorprV3VIAAAC/hHADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMhXADAABMpYXRBQBmVjStT5Ofo3PKziY/BwA0J6zcAAAAUyHcAAAAUyHcAAAAUyHcAAAAUyHcAAAAUyHcAAAAU7GLcDN//nwFBgbKxcVF/fr1U15e3gX7Ll26VBaLpc7h4uJiw2oBAIA9MzzcrFy5UklJSUpNTdW2bdsUEhKimJgYlZWVXXCMu7u7Dh8+bD0OHDhgw4oBAIA9M/xN/NLT05WQkKD4+HhJUmZmpt577z1lZWVp8uTJ9Y6xWCzy8/OzZZm4QryZHQDAVgxduamqqlJ+fr6io6OtbQ4ODoqOjtamTZsuOO7EiRPq0qWLAgICdMcdd+iLL764YN/KykqVl5fXOQAAgHkZGm6OHj2q6upq+fr61mn39fVVSUlJvWOCgoKUlZWlt99+W6+99ppqamrUv39/ffvtt/X2T0tLk4eHh/UICAho9HkAAAD7Yfiem0sVGRmpuLg4hYaGatCgQVq1apXat2+vl19+ud7+ycnJOnbsmPUoLi62ccUAAMCWDN1z4+3tLUdHR5WWltZpLy0tbfCempYtW+r666/Xvn376n3e2dlZzs7OV1wrAABoHgxduXFyclJYWJiys7OtbTU1NcrOzlZkZGSDXqO6ulo7d+5Uhw4dmqpMAADQjBh+t1RSUpLGjBmj8PBwRUREKCMjQxUVFda7p+Li4uTv76+0tDRJ0rRp0/TrX/9a3bt31w8//KDnnntOBw4c0P3332/kNAAAgJ0wPNzExsbqyJEjSklJUUlJiUJDQ7Vu3TrrJuOioiI5OPxvgen7779XQkKCSkpK1LZtW4WFhenzzz9XcHCwUVMAAAB2xPBwI0mJiYlKTEys97mcnJw6j+fMmaM5c+bYoCoAANAcNbu7pQAAAC6GcAMAAEyFcAMAAEyFcAMAAEyFcAMAAEyFcAMAAEyFcAMAAEyFcAMAAEyFcAMAAEzFLt6h2F6ETfxbk58j/7m4Jj8HAABXs8taubn55pv1ww8/nNdeXl6um2+++UprAgAAuGyXFW5ycnJUVVV1XvupU6f06aefXnFRAAAAl+uSLkv9+9//tn795ZdfqqSkxPq4urpa69atk7+/f+NVBwAAcIkuKdyEhobKYrHIYrHUe/mpVatWmjdvXqMVBwAAcKkuKdx88803qq2tVdeuXZWXl6f27dtbn3NycpKPj48cHR0bvUgAAICGuqRw06VLF0lSTU1NkxQDAABwpRocbtasWaMhQ4aoZcuWWrNmzUX73n777VdcGAAAwOVocLgZPny4SkpK5OPjo+HDh1+wn8ViUXV1dWPUBgAAcMkaHG5+eimKy1IAAMBeXfY7FGdnZys7O1tlZWV1wo7FYtGSJUsapTgAAIBLdVnhZurUqZo2bZrCw8PVoUMHWSyWxq4LAADgslxWuMnMzNTSpUt17733NnY9AAAAV+SyPn6hqqpK/fv3b+xaAAAArthlhZv7779fy5cvb+xaAAAArliDL0slJSVZv66pqdHChQv10UcfqW/fvmrZsmWdvunp6Y1XIQAAwCVocLgpKCio8zg0NFSStGvXrjrtbC4GAABGanC4+fjjj5uyDgAAgEZxWXtuAAAA7BXhBgAAmArhBgAAmIpdhJv58+crMDBQLi4u6tevn/Ly8ho0bsWKFbJYLBf9IE8AAHB1MTzcrFy5UklJSUpNTdW2bdsUEhKimJgYlZWVXXTc/v379Ze//EUDBw60UaUAAKA5MDzcpKenKyEhQfHx8QoODlZmZqZcXV2VlZV1wTHV1dUaPXq0pk6dqq5du9qwWgAAYO8MDTdVVVXKz89XdHS0tc3BwUHR0dHatGnTBcdNmzZNPj4+GjdunC3KBAAAzchlfXBmYzl69Kiqq6vl6+tbp93X11d79uypd8zGjRu1ZMkSbd++vUHnqKysVGVlpfVxeXn5ZdcLAADsn+GXpS7F8ePHde+992rRokXy9vZu0Ji0tDR5eHhYj4CAgCauEgAAGMnQlRtvb285OjqqtLS0Tntpaan8/PzO6//1119r//79GjZsmLWtpqZGktSiRQsVFhaqW7dudcYkJyfX+Vys8vJyAg4AACZmaLhxcnJSWFiYsrOzrbdz19TUKDs7W4mJief179mzp3bu3FmnbcqUKTp+/Ljmzp1bb2hxdnaWs7Nzk9QPAADsj6HhRjr7aeNjxoxReHi4IiIilJGRoYqKCsXHx0uS4uLi5O/vr7S0NLm4uOi6666rM97T01OSzmsHAABXJ8PDTWxsrI4cOaKUlBSVlJQoNDRU69ats24yLioqkoNDs9oaBAAADGR4uJGkxMTEei9DSVJOTs5Fxy5durTxCwIAAM0WSyIAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBUCDcAAMBU7CLczJ8/X4GBgXJxcVG/fv2Ul5d3wb6rVq1SeHi4PD091bp1a4WGhurVV1+1YbUAAMCeGR5uVq5cqaSkJKWmpmrbtm0KCQlRTEyMysrK6u3v5eWlJ598Ups2bdK///1vxcfHKz4+XuvXr7dx5QAAwB4ZHm7S09OVkJCg+Ph4BQcHKzMzU66ursrKyqq3f1RUlH7/+9+rV69e6tatmyZMmKC+fftq48aNNq4cAADYI0PDTVVVlfLz8xUdHW1tc3BwUHR0tDZt2vSL42tra5Wdna3CwkLdeOON9faprKxUeXl5nQMAAJhXCyNPfvToUVVXV8vX17dOu6+vr/bs2XPBcceOHZO/v78qKyvl6Oiov/71r7r11lvr7ZuWlqapU6c2at1XomhanyY/R+eUnU1+DgAA7JXhl6Uuh5ubm7Zv366tW7dqxowZSkpKUk5OTr19k5OTdezYMetRXFxs22IBAIBNGbpy4+3tLUdHR5WWltZpLy0tlZ+f3wXHOTg4qHv37pKk0NBQ7d69W2lpaYqKijqvr7Ozs5ydnRu1bgAAYL8MXblxcnJSWFiYsrOzrW01NTXKzs5WZGRkg1+npqZGlZWVTVEiAABoZgxduZGkpKQkjRkzRuHh4YqIiFBGRoYqKioUHx8vSYqLi5O/v7/S0tIknd1DEx4erm7duqmyslJr167Vq6++qgULFhg5DQAAYCcMDzexsbE6cuSIUlJSVFJSotDQUK1bt866ybioqEgODv9bYKqoqNBDDz2kb7/9Vq1atVLPnj312muvKTY21qgpAAAAO2J4uJGkxMREJSYm1vvczzcKP/PMM3rmmWdsUBUAAGiOmuXdUgAAABdCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZCuAEAAKZiF+Fm/vz5CgwMlIuLi/r166e8vLwL9l20aJEGDhyotm3bqm3btoqOjr5ofwAAcHUxPNysXLlSSUlJSk1N1bZt2xQSEqKYmBiVlZXV2z8nJ0d33XWXPv74Y23atEkBAQEaPHiwDh48aOPKAQCAPTI83KSnpyshIUHx8fEKDg5WZmamXF1dlZWVVW//119/XQ899JBCQ0PVs2dPLV68WDU1NcrOzrZx5QAAwB4ZGm6qqqqUn5+v6Ohoa5uDg4Oio6O1adOmBr3GyZMndfr0aXl5eTVVmQAAoBlpYeTJjx49qurqavn6+tZp9/X11Z49exr0GpMmTVLHjh3rBKSfqqysVGVlpfVxeXn55RcMAADsnuGXpa7ErFmztGLFCr311ltycXGpt09aWpo8PDysR0BAgI2rBAAAtmRouPH29pajo6NKS0vrtJeWlsrPz++iY59//nnNmjVLH3zwgfr27XvBfsnJyTp27Jj1KC4ubpTaAQCAfTI03Dg5OSksLKzOZuBzm4MjIyMvOO7ZZ5/V9OnTtW7dOoWHh1/0HM7OznJ3d69zAAAA8zJ0z40kJSUlacyYMQoPD1dERIQyMjJUUVGh+Ph4SVJcXJz8/f2VlpYmSZo9e7ZSUlK0fPlyBQYGqqSkRJLUpk0btWnTxrB5AAAA+2B4uImNjdWRI0eUkpKikpIShYaGat26ddZNxkVFRXJw+N8C04IFC1RVVaU//vGPdV4nNTVVTz/9tC1LBwAAdsjwcCNJiYmJSkxMrPe5nJycOo/379/f9AUBAIBmq1nfLQUAAPBzhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhBsAAGAqhoeb+fPnKzAwUC4uLurXr5/y8vIu2PeLL77QH/7wBwUGBspisSgjI8N2hQIAgGbB0HCzcuVKJSUlKTU1Vdu2bVNISIhiYmJUVlZWb/+TJ0+qa9eumjVrlvz8/GxcLQAAaA4MDTfp6elKSEhQfHy8goODlZmZKVdXV2VlZdXb/4YbbtBzzz2nO++8U87OzjauFgAANAeGhZuqqirl5+crOjr6f8U4OCg6OlqbNm1qtPNUVlaqvLy8zgEAAMzLsHBz9OhRVVdXy9fXt067r6+vSkpKGu08aWlp8vDwsB4BAQGN9toAAMD+GL6huKklJyfr2LFj1qO4uNjokgAAQBNqYdSJvb295ejoqNLS0jrtpaWljbpZ2NnZmf05AABcRQxbuXFyclJYWJiys7OtbTU1NcrOzlZkZKRRZQEAgGbOsJUbSUpKStKYMWMUHh6uiIgIZWRkqKKiQvHx8ZKkuLg4+fv7Ky0tTdLZTchffvml9euDBw9q+/btatOmjbp3727YPAAAgP0wNNzExsbqyJEjSklJUUlJiUJDQ7Vu3TrrJuOioiI5OPxvcenQoUO6/vrrrY+ff/55Pf/88xo0aJBycnJsXT4AALBDhoYbSUpMTFRiYmK9z/08sAQGBqq2ttYGVQEAgObK9HdLAQCAqwvhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmArhBgAAmIpdhJv58+crMDBQLi4u6tevn/Ly8i7a/80331TPnj3l4uKiPn36aO3atTaqFAAA2DvDw83KlSuVlJSk1NRUbdu2TSEhIYqJiVFZWVm9/T///HPdddddGjdunAoKCjR8+HANHz5cu3btsnHlAADAHhkebtLT05WQkKD4+HgFBwcrMzNTrq6uysrKqrf/3Llz9dvf/lYTJ05Ur169NH36dP3qV7/SSy+9ZOPKAQCAPTI03FRVVSk/P1/R0dHWNgcHB0VHR2vTpk31jtm0aVOd/pIUExNzwf4AAODq0sLIkx89elTV1dXy9fWt0+7r66s9e/bUO6akpKTe/iUlJfX2r6ysVGVlpfXxsWPHJEnl5eXn9a2u/PGS6r8cx1tWN/k56pvbxTDvpsO8mw7zbhjm3XSYd9Opb97n2mpra39xvKHhxhbS0tI0derU89oDAgIMqEa6zhYnSfOwxVkuCfNuQszbbjDvJsS87YbR8z5+/Lg8PC7+fTE03Hh7e8vR0VGlpaV12ktLS+Xn51fvGD8/v0vqn5ycrKSkJOvjmpoafffdd2rXrp0sFssVzuDSlJeXKyAgQMXFxXJ3d7fpuY3EvJn31YB5M++rgZHzrq2t1fHjx9WxY8df7GtouHFyclJYWJiys7M1fPhwSWfDR3Z2thITE+sdExkZqezsbD366KPWtg8//FCRkZH19nd2dpazs3OdNk9Pz8Yo/7K5u7tfVf8xnMO8ry7M++rCvK8uRs37l1ZszjH8slRSUpLGjBmj8PBwRUREKCMjQxUVFYqPj5ckxcXFyd/fX2lpaZKkCRMmaNCgQXrhhRd02223acWKFfrXv/6lhQsXGjkNAABgJwwPN7GxsTpy5IhSUlJUUlKi0NBQrVu3zrppuKioSA4O/7upq3///lq+fLmmTJmiJ554Qj169NDq1at13XU2uQoIAADsnOHhRpISExMveBkqJyfnvLaRI0dq5MiRTVxV43N2dlZqaup5l8nMjnkz76sB82beV4PmMm9LbUPuqQIAAGgmDH+HYgAAgMZEuAEAAKZCuAEAAKZCuLGBTz75RMOGDVPHjh1lsVi0evVqo0uyibS0NN1www1yc3OTj4+Phg8frsLCQqPLanILFixQ3759re8DERkZqffff9/osmxu1qxZslgsdd6TyoyefvppWSyWOkfPnj2NLssmDh48qHvuuUft2rVTq1at1KdPH/3rX/8yuqwmFRgYeN7P22KxaPz48UaX1qSqq6v11FNP6ZprrlGrVq3UrVs3TZ8+vUEfhWAEu7hbyuwqKioUEhKisWPHasSIEUaXYzO5ubkaP368brjhBp05c0ZPPPGEBg8erC+//FKtW7c2urwm06lTJ82aNUs9evRQbW2tli1bpjvuuEMFBQXq3bu30eXZxNatW/Xyyy+rb9++RpdiE71799ZHH31kfdyihfn/1/r9999rwIABuummm/T++++rffv2+uqrr9S2bVujS2tSW7duVXX1/z5badeuXbr11lub5R28l2L27NlasGCBli1bpt69e+tf//qX4uPj5eHhoUceecTo8s5j/v8C7cCQIUM0ZMgQo8uwuXXr1tV5vHTpUvn4+Cg/P1833nijQVU1vWHDhtV5PGPGDC1YsECbN2++KsLNiRMnNHr0aC1atEjPPPOM0eXYRIsWLS74ETBmNXv2bAUEBOiVV16xtl1zzTUGVmQb7du3r/N41qxZ6tatmwYNGmRQRbbx+eef64477tBtt90m6ewK1t///nfl5eUZXFn9uCwFmzn3iexeXl4GV2I71dXVWrFihSoqKi74ESFmM378eN12222Kjo42uhSb+eqrr9SxY0d17dpVo0ePVlFRkdElNbk1a9YoPDxcI0eOlI+Pj66//notWrTI6LJsqqqqSq+99prGjh1r888qtLX+/fsrOztbe/fulSTt2LFDGzdutNt/uLNyA5uoqanRo48+qgEDBlwV7ya9c+dORUZG6tSpU2rTpo3eeustBQcHG11Wk1uxYoW2bdumrVu3Gl2KzfTr109Lly5VUFCQDh8+rKlTp2rgwIHatWuX3NzcjC6vyfznP//RggULlJSUpCeeeEJbt27VI488IicnJ40ZM8bo8mxi9erV+uGHH3TfffcZXUqTmzx5ssrLy9WzZ085OjqqurpaM2bM0OjRo40urV6EG9jE+PHjtWvXLm3cuNHoUmwiKChI27dv17Fjx/SPf/xDY8aMUW5urqkDTnFxsSZMmKAPP/xQLi4uRpdjMz/9l2vfvn3Vr18/denSRW+88YbGjRtnYGVNq6amRuHh4Zo5c6Yk6frrr9euXbuUmZl51YSbJUuWaMiQIQ36lOrm7o033tDrr7+u5cuXq3fv3tq+fbseffRRdezY0S5/3oQbNLnExES9++67+uSTT9SpUyejy7EJJycnde/eXZIUFhamrVu3au7cuXr55ZcNrqzp5Ofnq6ysTL/61a+sbdXV1frkk0/00ksvqbKyUo6OjgZWaBuenp669tprtW/fPqNLaVIdOnQ4L6z36tVL//znPw2qyLYOHDigjz76SKtWrTK6FJuYOHGiJk+erDvvvFOS1KdPHx04cEBpaWmEG1xdamtr9fDDD+utt95STk7OVbHZ8EJqampUWVlpdBlN6pZbbtHOnTvrtMXHx6tnz56aNGnSVRFspLMbqr/++mvde++9RpfSpAYMGHDeWzvs3btXXbp0Magi23rllVfk4+Nj3WBrdidPnqzzIdaS5OjoqJqaGoMqujjCjQ2cOHGizr/ivvnmG23fvl1eXl7q3LmzgZU1rfHjx2v58uV6++235ebmppKSEkmSh4eHWrVqZXB1TSc5OVlDhgxR586ddfz4cS1fvlw5OTlav3690aU1KTc3t/P2U7Vu3Vrt2rUz9T6rv/zlLxo2bJi6dOmiQ4cOKTU1VY6OjrrrrruMLq1JPfbYY+rfv79mzpypUaNGKS8vTwsXLtTChQuNLq3J1dTU6JVXXtGYMWOuitv+pbN3gc6YMUOdO3dW7969VVBQoPT0dI0dO9bo0upXiyb38ccf10o67xgzZozRpTWp+uYsqfaVV14xurQmNXbs2NouXbrUOjk51bZv3772lltuqf3ggw+MLssQgwYNqp0wYYLRZTSp2NjY2g4dOtQ6OTnV+vv718bGxtbu27fP6LJs4p133qm97rrrap2dnWt79uxZu3DhQqNLson169fXSqotLCw0uhSbKS8vr50wYUJt586da11cXGq7du1a++STT9ZWVlYaXVq9+FRwAABgKrzPDQAAMBXCDQAAMBXCDQAAMBXCDQAAMBXCDQAAMBXCDQAAMBXCDQAAMBXCDQAAMBXCDYCrWmBgoDIyMowuA0AjItwAsBtRUVF69NFHjS4DQDNHuAEAAKZCuAFgF+677z7l5uZq7ty5slgsslgs2r9/v3JzcxURESFnZ2d16NBBkydP1pkzZ6zjoqKilJiYqMTERHl4eMjb21tPPfWULvdj8xYvXixPT09lZ2c31tQA2BjhBoBdmDt3riIjI5WQkKDDhw/r8OHDatmypYYOHaobbrhBO3bs0IIFC7RkyRI988wzdcYuW7ZMLVq0UF5enubOnav09HQtXrz4kmt49tlnNXnyZH3wwQe65ZZbGmtqAGyshdEFAIAkeXh4yMnJSa6urvLz85MkPfnkkwoICNBLL70ki8Winj176tChQ5o0aZJSUlLk4HD232cBAQGaM2eOLBaLgoKCtHPnTs2ZM0cJCQkNPv+kSZP06quvKjc3V717926SOQKwDVZuANit3bt3KzIyUhaLxdo2YMAAnThxQt9++6217de//nWdPpGRkfrqq69UXV3doPO88MILWrRokTZu3EiwAUyAcAPgqjdw4EBVV1frjTfeMLoUAI2AcAPAbjg5OdVZbenVq5c2bdpUZ3PwZ599Jjc3N3Xq1MnatmXLljqvs3nzZvXo0UOOjo4NOm9ERITef/99zZw5U88///wVzgKA0Qg3AOxGYGCgtmzZov379+vo0aN66KGHVFxcrIcfflh79uzR22+/rdTUVCUlJVn320hSUVGRkpKSVFhYqL///e+aN2+eJkyYcEnn7t+/v9auXaupU6fypn5AM8eGYgB24y9/+YvGjBmj4OBg/fjjj/rmm2+0du1aTZw4USEhIfLy8tK4ceM0ZcqUOuPi4uL0448/KiIiQo6OjpowYYIeeOCBSz7/b37zG7333nsaOnSoHB0d9fDDDzfW1ADYkKX2ct8MAgDsQFRUlEJDQ1ltAWDFZSkAAGAqhBsApvXpp5+qTZs2FzwAmBOXpQCY1o8//qiDBw9e8Pnu3bvbsBoAtkK4AQAApsJlKQAAYCqEGwAAYCqEGwAAYCqEGwAAYCqEGwAAYCqEGwAAYCqEGwAAYCqEGwAAYCr/D7Xhz2EAc5v9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.barplot(x='top_k', y='hit', hue='step_back_prompting', data=hit_stat_df, errorbar=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T02:59:32.745866Z",
     "iopub.status.busy": "2024-10-08T02:59:32.744564Z",
     "iopub.status.idle": "2024-10-08T02:59:32.758186Z",
     "shell.execute_reply": "2024-10-08T02:59:32.755832Z",
     "shell.execute_reply.started": "2024-10-08T02:59:32.745790Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "ollama_llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:00:28.526130Z",
     "iopub.status.busy": "2024-10-08T03:00:28.525697Z",
     "iopub.status.idle": "2024-10-08T03:00:28.535149Z",
     "shell.execute_reply": "2024-10-08T03:00:28.533176Z",
     "shell.execute_reply.started": "2024-10-08T03:00:28.526093Z"
    }
   },
   "outputs": [],
   "source": [
    "def rag(question, n_chunks=3):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{question}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "\n",
    "    step_back_retrieve_chain = get_retrieve_chain(n_chunks)\n",
    "    chunks = step_back_retrieve_chain.invoke(question)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{question}}', question)\n",
    "\n",
    "    return ollama_llm(prompt), chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:00:29.150488Z",
     "iopub.status.busy": "2024-10-08T03:00:29.150204Z",
     "iopub.status.idle": "2024-10-08T03:00:31.795450Z",
     "shell.execute_reply": "2024-10-08T03:00:31.794500Z",
     "shell.execute_reply.started": "2024-10-08T03:00:29.150464Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseLLM.__call__` was deprecated in langchain-core 0.1.7 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023年10月美国ISM制造业PMI指数为50.4%，较前一个月的55.5%明显回落。\n"
     ]
    }
   ],
   "source": [
    "print(rag('2023年10月美国ISM制造业PMI指数较上月有何变化？')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:00:35.200357Z",
     "iopub.status.busy": "2024-10-08T03:00:35.199582Z",
     "iopub.status.idle": "2024-10-08T03:00:35.222811Z",
     "shell.execute_reply": "2024-10-08T03:00:35.220558Z",
     "shell.execute_reply.started": "2024-10-08T03:00:35.200289Z"
    }
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(prediction_df, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:00:35.615595Z",
     "iopub.status.busy": "2024-10-08T03:00:35.614771Z",
     "iopub.status.idle": "2024-10-08T03:06:52.162344Z",
     "shell.execute_reply": "2024-10-08T03:06:52.161435Z",
     "shell.execute_reply.started": "2024-10-08T03:00:35.615525Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de8a174df1e3469f9f7fdf2a0efea32e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_df = predict(prediction_df, n_chunks=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:07:02.378021Z",
     "iopub.status.busy": "2024-10-08T03:07:02.377742Z",
     "iopub.status.idle": "2024-10-08T03:07:02.402186Z",
     "shell.execute_reply": "2024-10-08T03:07:02.401322Z",
     "shell.execute_reply.started": "2024-10-08T03:07:02.377997Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "import time\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        result = judge_llm.invoke(prompt).content\n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:07:03.074110Z",
     "iopub.status.busy": "2024-10-08T03:07:03.073763Z",
     "iopub.status.idle": "2024-10-08T03:09:39.815310Z",
     "shell.execute_reply": "2024-10-08T03:09:39.812970Z",
     "shell.execute_reply.started": "2024-10-08T03:07:03.074081Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc780707711e4537817972e989f6ec18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_df['raw_score'] = evaluate(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:09:39.816300Z",
     "iopub.status.busy": "2024-10-08T03:09:39.816167Z",
     "iopub.status.idle": "2024-10-08T03:09:39.819693Z",
     "shell.execute_reply": "2024-10-08T03:09:39.819272Z",
     "shell.execute_reply.started": "2024-10-08T03:09:39.816287Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['是', '否'], dtype=object)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['raw_score'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:09:39.820192Z",
     "iopub.status.busy": "2024-10-08T03:09:39.820072Z",
     "iopub.status.idle": "2024-10-08T03:09:39.830285Z",
     "shell.execute_reply": "2024-10-08T03:09:39.829876Z",
     "shell.execute_reply.started": "2024-10-08T03:09:39.820181Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:09:39.831223Z",
     "iopub.status.busy": "2024-10-08T03:09:39.831068Z",
     "iopub.status.idle": "2024-10-08T03:09:39.835376Z",
     "shell.execute_reply": "2024-10-08T03:09:39.834862Z",
     "shell.execute_reply.started": "2024-10-08T03:09:39.831209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.56"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
